/*
 *  ExtractorWrapperTest.java
 *
 *  Chau, Sau Ming
 *
 *  August 14, 2000
 *
 *  This is a demontration program of Extractor Wrapper (consists of
 *  DocumentMemory.java and StopMemory.java).  The flow of this
 *  program is repeatedly asking web page URL or text file name and
 *  summarizing the document in separate thread until user enters
 *  'q' to quit.  StopMemory is shared among threads.
 *
 *  For details of Extractor Wrapper, please refer to DocumentMemory.java
 *  and StopMemory.java.
 */
 
/*
 *  Modified by Peter Turney
 *
 *  August 10, 2001
 */

import java.util.*;
import java.io.*;
import java.net.*;

public class ExtractorWrapperTest extends Thread
{
  public static Counter thread_counter; // counter on number of thread created
  private static StopMemory StopMem;    // the shared StopMemory
  private DocumentMemory DocMem;        // DoccumetMemory for each thread
  
  private String str_file;    // path and file name of file to be summarized
  private String str_url;     // URL to be summarized
  private String str_output_file; // path and file name of HTML output file
  private int int_choice;     // choice of web page or file summarization
  private int max_num_byte_read; // maximum number of bytes to be read

  static
  { // Load Extractor dll
    System.loadLibrary("Extractor");
  }

  
  /*
   *  main() function of the test program.  Keep gathering info on the
   *  web page or text file, and create new thread to process each request
   *  until user type 'q' to quit the test program.
   */
  public static void main(String[] args)
  {
    try
    {
      boolean bln_continue = true;

      ExtractorWrapperTest extr_wrapper_test;
      String s;
      BufferedReader br = new BufferedReader(new InputStreamReader(System.in));

      System.out.println();
      System.out.println("  Welcome to Extractor Wrapper Demonstration Program");
      System.out.println("  --------------------------------------------------");
      while (bln_continue)
      {
        extr_wrapper_test = new ExtractorWrapperTest(); // Initialization
        extr_wrapper_test.start();  // Start a new thread to summarize
        System.out.println();
        System.out.print("    Enter 'c' to continue or 'q' to quit (default is 'q'): ");
        s = br.readLine();
        if (! s.equals("c"))
        {
          bln_continue = false;
        }
      }
    }
    catch (Exception e)
    {
    }
    finally
    { // Must clear StopMemory to prevent memory leak
      ExtractorWrapperTest.ClearStopMemory();
    }
  }


  /*
   *  Constructor that collects all info (e.g. web page URL or text file path
   *  and name) about the document to be summarized.  It also increment the
   *  thread counter (thread_counter).
   */
  public ExtractorWrapperTest()
  {
    if (thread_counter == null)
    { // Create thread_counter if not yet constructed.
      thread_counter = new Counter();
    }
    if (StopMem == null)
    { // Create StopMemory if not yet constructed.
      StopMem = new StopMemory();
    }
    // test adding stop words as strings
    String[] StopWords = {"a", "did", "has"};
    for (int i = 0; i < StopWords.length; i++) 
    { 
      int ErrorCode = StopMem.AddStopWord(1, 1, StopWords[i]);
      if (ErrorCode != 0) 
      {
      	System.out.println("    ERROR when adding stop words as strings!");
        System.out.println(StopMem.GetErrorMessage(ErrorCode));
        System.exit(1);
      }
    }
    // test adding stop words as byte arrays
    for (int i = 0; i < StopWords.length; i++) 
    { 
      int ErrorCode = StopMem.AddStopWord(1, 1, StopWords[i].getBytes());
      if (ErrorCode != 0) 
      {
      	System.out.println("    ERROR when adding stop words as byte arrays!");
        System.out.println(StopMem.GetErrorMessage(ErrorCode));
        System.exit(1);
      }
    }
    // test adding stop phrases as strings
    String[] StopPhrases = {"a did", "did the", "has it"};
    for (int i = 0; i < StopPhrases.length; i++) 
    { 
      int ErrorCode = StopMem.AddStopPhrase(1, 1, StopPhrases[i]);
      if (ErrorCode != 0) 
      {
      	System.out.println("    ERROR when adding stop phrases as strings!");
        System.out.println(StopMem.GetErrorMessage(ErrorCode));
        System.exit(1);
      }
    }
    // test adding stop phrases as byte arrays
    for (int i = 0; i < StopPhrases.length; i++) 
    { 
      int ErrorCode = StopMem.AddStopPhrase(1, 1, StopPhrases[i].getBytes());
      if (ErrorCode != 0) 
      {
      	System.out.println("    ERROR when adding stop phrases as byte arrays!");
        System.out.println(StopMem.GetErrorMessage(ErrorCode));
        System.exit(1);
      }
    }
    max_num_byte_read = 80000;
    thread_counter.inc();  // Increment thread counter
    BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
    String str_user_input;  // String for user input
    str_file = "";
    str_url = "";
    try
    {
      System.out.println();
      System.out.println("    Please select one of the following");
      System.out.println();
      System.out.println("      1. Summarize a web page");
      System.out.println("      2. Summarize a text file");
      System.out.println();
      System.out.print("    Enter the selection (default is 1): ");
      str_user_input = br.readLine();
      if ((! str_user_input.equals("1")) && (! str_user_input.equals("2")))
      { // default selection
      	str_user_input = "1";
      }
      if (str_user_input.equals("1"))
      {
        System.out.println();
        System.out.print("    Enter the URL of a web page (default is extractor.iit.nrc.ca): ");
        str_url = br.readLine();
        if (str_url.equals(""))
        { // default URL
          str_url = "http://extractor.iit.nrc.ca/";
        }
        int_choice = 1;
      }
      else if (str_user_input.equals("2"))
      {
        System.out.println();
        System.out.print("    Enter the path and file name for input: ");
        str_file = br.readLine();
        int_choice = 2;
      }
      System.out.println();
      System.out.print("    Enter file name for output (default is Extractor_Result.htm): ");
      str_output_file = br.readLine();
      if (str_output_file.equals(""))
      {
        str_output_file = "Extractor_Result.htm";
      }
    }
    catch (Exception e)
    {
    }
  }


  /*
   *  Clear StopMemory.  Before clearing, it must wait until all processing
   *  threads finish (i.e. thread_counter must be 0); otherwise, abnormal
   *  termination may occur.
   */
  public static void ClearStopMemory()
  {
    try
    {
      if (thread_counter.value() > 0)
      {
        thread_counter.wait_for_threads_finish();
      }
    }
    catch (Exception e)
    {
      e.printStackTrace();
    }
    finally
    {
      if (StopMem != null)
      {
        StopMem.ClearStopMemory();
      }
    }
  }


  /*
   *  Summarize the web page of text file in separate thread.
   */
  public void run()
  {
    DocMem = new DocumentMemory(StopMem);
    byte[] byte_content = new byte[0];  // content of web page or text file

    try
    {
      DocMem.ActivateHighlights();
      DocMem.SetHighlightType(1 + 2 + 4);
        // Check DocumentMemory.java for the choice of HighlightType

      if (int_choice == 1)
      { // Summarize a web page
        DocMem.ActivateHTMLFilter();
        byte_content = get_web_page_content();
      }
      else
      {
        byte_content = get_text_file_content();
      }
      
      FileOutputStream file_out = new FileOutputStream(str_output_file);
      byte[] byte_nohtml_content = CodeDetector.convert_html_to_nohtml(byte_content);
      
      int CharCodeID = CodeDetector.detect_code(byte_content, byte_nohtml_content);
      int LanguageID = 0;
      
      DocMem.SetInputCode(CharCodeID);
      DocMem.SetOutputCode(CharCodeID);
      if (CharCodeID > 2)
      {
        LanguageID = 3;
        DocMem.SetDocumentLanguage(LanguageID);
      }
      DocMem.ReadDocumentBuffer(byte_content);
      DocMem.SignalDocumentEnd();
      LanguageID = DocMem.GetDocumentLanguage();

      file_out.write((new String("<html>")).getBytes());
      file_out.write(13); file_out.write(10);
      file_out.write((new String("<head>")).getBytes());
      file_out.write(13); file_out.write(10);
      file_out.write((new String("<title>Extractor Result</title>")).getBytes());
      file_out.write(13); file_out.write(10);
      if (LanguageID == 3)
      {
        file_out.write((new String("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=iso-2022-jp\">")).getBytes());
      }
      else
      {
        file_out.write((new String("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=iso-8859-1\">")).getBytes());
      }
      file_out.write(13); file_out.write(10);
      file_out.write((new String("</head>")).getBytes());
      file_out.write(13); file_out.write(10);
      file_out.write((new String("<body>")).getBytes());
      file_out.write(13); file_out.write(10);
      if (int_choice == 1)
      { 
        file_out.write(
          (new String("<b><FONT SIZE=\"4\">Web Page Summarized: </b></FONT>" +
           str_url + "<br>")).getBytes());
        file_out.write(13); file_out.write(10);
      }
      else
      {
        file_out.write(
          (new String("<b><FONT SIZE=\"4\">Text File Summarized: </b></FONT>" +
           str_file + "<br>")).getBytes());
        file_out.write(13); file_out.write(10);
      }
      
      file_out.write((new String("<br><b><FONT SIZE=\"4\">Language: </b></FONT>")).getBytes());
      
      switch (LanguageID)
      {
        case 1:
          file_out.write((new String("English <br>")).getBytes());
          break;
        case 2:
          file_out.write((new String("French <br>")).getBytes());
          break;
        case 3:
          file_out.write((new String("Japanese <br>")).getBytes());
          break;
        case 4:
          file_out.write((new String("German <br>")).getBytes());
          break;
        case 5:
          file_out.write((new String("Spanish <br>")).getBytes());
          break;
        default:
          file_out.write((new String("Unknown <br>")).getBytes());
          break;
      }
      file_out.write(13); file_out.write(10);
      file_out.write((new String("<br><b><FONT SIZE=\"4\">Keyphrases:</b></FONT><br>")).getBytes());
      
      // Get the keyphrases
      int PhraseListSize = DocMem.GetPhraseListSize();
      String str_phrase = "";
      file_out.write(13); file_out.write(10);
      for (int PhraseIndex = 0; PhraseIndex < PhraseListSize; PhraseIndex++)
      {
        byte[] temp = DocMem.GetPhraseByIndex(PhraseIndex);
        file_out.write((new String("<br>&nbsp;&nbsp;&nbsp;-&nbsp;")).getBytes());
        file_out.write(temp);
        file_out.write(13); file_out.write(10);
      }

      file_out.write((new String("<br><br><br><FONT SIZE=\"4\"><b>Highlights:</b></FONT><br>")).getBytes());
      file_out.write(13); file_out.write(10);
      
      // Get the Highlights
      int HighlightListSize = DocMem.GetHighlightListSize();
      String str_highlight = "";
      for (int HighlightIndex = 0; HighlightIndex < HighlightListSize; HighlightIndex++)
      {
        file_out.write((new String("<br>&nbsp;&nbsp;&nbsp;-&nbsp;")).getBytes());
        file_out.write(DocMem.GetHighlightByIndex(HighlightIndex));
        file_out.write(13); file_out.write(10);
      }
      file_out.write((new String("</body>")).getBytes());
      file_out.write(13); file_out.write(10);
      file_out.write((new String("</html>")).getBytes());
      file_out.write(13); file_out.write(10);
      file_out.close();
    }
    catch (Exception e)
    {
      e.printStackTrace();
    }
    finally
    {
      DocMem.ClearDocumentMemory(); // Clear DocumentMemory
      thread_counter.dec(); // Decrement thread counter
    }
  }


  /*
   *  Return the content of the text file.  Retrieve content in bytes rather
   *  than character (char) is strongly recommended as character type in Java
   *  is 2 bytes long.  Conversion from char to byte in Java may result
   *  lost of precision.
   */
  public byte[] get_text_file_content()
  {
    byte[] byte_content = new byte[0];
    try
    {
      if (str_file.equals(""))
      { // Default text file
        str_file = new String("script.html");
        //str_file = new String("Extractor.htm");
      }
      FileReader file_in = new FileReader(str_file);
      int in = 0;
      int counter = 0;
      byte[] byte_temp_content = new byte[max_num_byte_read];
      while (((in  = file_in.read()) != -1) && (counter < max_num_byte_read))
      {
        byte_temp_content[counter] = (byte)in;
        counter++;
      }
      byte_content = new byte[counter];
      System.arraycopy(byte_temp_content, 0, byte_content, 0, counter);
      file_in.close();  // close the config file stream*/
    }
    catch (Exception e)
    {
      e.printStackTrace();
    }
    return byte_content;
  }


  /*
   *  Return the content of the web page.  Retrieve content in bytes rather
   *  than character (char) is strongly recommended as character type in Java
   *  is 2 bytes long.  Conversion from char to byte in Java may result
   *  lost of precision.
   */
  public byte[] get_web_page_content()
  {
    byte[] byte_content = new byte[0];
    try
    {
      URL url;
      HttpURLConnection httpcon;
      InputStream is_http = null;
      BufferedReader br_http = null;
      byte[] byte_temp_content = new byte[max_num_byte_read];

      if (str_url.equals(""))
      { // Default web page.
        //str_url = new String("http://Extractor.iit.nrc.ca/"); // English
        //str_url = new String("http://bizit.nikkeibp.co.jp/it/top/bizit.shtml"); // Japanese
        //str_url = new String("http://www.kdd.co.jp/service/etempo/outline.html"); // Japanese
        //str_url = new String("http://ostracon.biologie.uni-kl.de/b_online/d42/42c.htm"); // German
        str_url = new String("http://www.el-mundo.es/noticias/"); // Spanish
      }
      
      url = new URL(str_url);
      httpcon = (HttpURLConnection)url.openConnection();  // connection to the web page
      is_http = httpcon.getInputStream(); // Get the input stream of the HttpURLConnection.

      if (is_http != null)
      { // Retrieve content of the web page
        int in;
        int counter = 0;
        while (((in = is_http.read()) != -1) && (counter < max_num_byte_read))
        { // Continue get character until -1 is fetched.
          byte_temp_content[counter] = (byte)in;
          counter++;
        }
        is_http.close(); // Close the buffered reader
        byte_content = new byte[counter];
        System.arraycopy(byte_temp_content, 0, byte_content, 0, counter);
      }
      httpcon.disconnect();
    }
    catch (Exception e)
    {
      e.printStackTrace();
    }
    return byte_content;
  }
}


/*
 *  classs Counter
 *
 *  The Counter class is designed for multi-threading environment.
 *
 */
class Counter
{
  int int_counter;
  
  /* 
   *  Constructor
   */
  public Counter()
  {
    int_counter = 0;
  }

  
  /* 
   *  Increment counter by 1
   */
  public synchronized void inc()
  {
    int_counter++;
  }

  
  /* 
   *  Decrement counter by 1
   */
  public synchronized void dec()
  {
    int_counter--;
    if (int_counter == 0)
    {
      notifyAll();
    }
  }

  
  /* 
   *  Return the current counter value
   */
  public synchronized int value()
  {
    return int_counter;
  }
  
  
  /* 
   *  Wait for other threads call dec() in that NotifyAll() is called.
   */
  public synchronized void wait_for_threads_finish()
  {
    try
    {
      wait();
    }
    catch (Exception e)
    {
    }
  }
}